/**
 * 
 */
package com.gragra.frameStatistics;
import java.util.Arrays;
import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntCollection;
import it.unimi.dsi.fastutil.ints.IntList;
import it.unimi.dsi.fastutil.ints.IntSet;
import com.gragra.data.Corpus;
import com.gragra.sampling.sequentialStructure.WordAndTagHolder;
import com.gragra.util.DeepIntNumberer;
import com.gragra.util.StringToInt;
/**
 * this can be used to read corpus of WordAndTagHolders and creates a set of statistics; the main statistics
 * concern frames that have an equal number of words on both side of the empty element; the empty element is 
 * represented by a -2; the statistics can used sentence end and beginning markers that are indicated by a -1;
 * when useTags true is given, then tags are used for frames, otherwise the words are used; the constructor that
 * has no useTags variable sets it to false; if a word has no tag (indicated by the WordAndTagHolder returning
 * a number < 0 when queried for the tag at that position) the word is used instead; frames are coded by each
 * instance independently
 * @author Christoph Teichmann
 * created Mar 24, 2013 11:48:07 AM
 * @version 0.1
 */
public class StatisticsGenerator
{
	/**
	 * 
	 * @param word
	 * @return
	 */
	public IntSet getFramesForWord(int word)
	{
		Int2DoubleOpenHashMap il = this.reverseCoCounts.get(word);
		return il == null ? null : il.keySet();
	}
	/**
	 * 
	 */
	private final Int2ObjectOpenHashMap<Int2DoubleOpenHashMap> reverseCoCounts = new Int2ObjectOpenHashMap<Int2DoubleOpenHashMap>();
	/**
	 * this map contains the counts of the co-occurrences between the frames and the words that occur between
	 * them; first int is a number that identifies the frame being used; the second int is the word code; double
	 * is the number of co-occurrences
	 */
	private final Int2ObjectOpenHashMap<Int2DoubleOpenHashMap> coCounts = new Int2ObjectOpenHashMap<Int2DoubleOpenHashMap>();
	/**
	 * mapping from frames (array of ints with a -2 in the middle) to int codes
	 */
	private final DeepIntNumberer frameNumbers = new DeepIntNumberer();
	/**
	 * this map can be used to look up the int[] that is mapped to a certain number
	 */
	private final Int2ObjectOpenHashMap<int[]> reverseFrames = new Int2ObjectOpenHashMap<int[]>();
	/**
	 * this maps the number code for a frame to the number of times it has occurred, by the semantics that are
	 * used in the fastUtils framework a frame that has not been added to the map will return 0.0
	 */
	private final Int2DoubleOpenHashMap frameCounts = new Int2DoubleOpenHashMap();
	/**
	 * contain the number of times a word has occurred in the corpus, the word is represented by it's number code
	 * as given by the StringToInt class
	 */
	private final Int2DoubleOpenHashMap wordCounts = new Int2DoubleOpenHashMap();
	/**
	 * contains the current mapping of the sentence which will be used for the frames, padded with -1 as appropriate
	 */
	private final IntList mappedSentence = new IntArrayList();
	/**
	 * contains the words of the current sentence, padded with -1 as appropriate
	 */
	private final IntList wordSentence = new IntArrayList();
	/**
	 * this array is used to hold the frame that is currently being worked on
	 */
	private final int[] frame;
	/**
	 * the amount of pre-sentence padding that is used
	 */
	private final int usePreMarker;
	/**
	 * the amount of post-sentence padding that is used
	 */
	private final int usePostMarker;
	/**
	 * stores the statistics from the given corpus, the given size of the frame (which must be 2*x+1 for some x)
	 * and the given number of pre- and post-sentence padding indicators; tags will not be used
	 * @param c corpus used
	 * @param frameSize the complete size of the frame used
	 * @param usePreMarker the number of pre-sentence padding markers used
	 * @param usePostMarker the number of post-sentence padding markers used
	 */
	public StatisticsGenerator(Corpus<WordAndTagHolder> c, int frameSize,
							int usePreMarker,int usePostMarker)
	{this(c, frameSize, usePreMarker, usePostMarker, false);}
	/**
	 * this creates a new instance and stores its statistics
	 * @param c the corpus that is used to create the statistics
	 * @param frameSize the size of the frame used
	 * @param usePreMarker the number of pre-sentence markers
	 * @param usePostMarker the number of post-sentence markers
	 * @param useTags whether to replace words with their tag (if present)
	 */
	public StatisticsGenerator(Corpus<WordAndTagHolder> c, int frameSize,
							int usePreMarker,int usePostMarker,boolean useTags)
	{
		if(frameSize % 2 != 1)
		{throw new IllegalArgumentException("framesizes need to be (2*x)+1 for some value of x");}
		this.frame = new int[frameSize];
		this.usePreMarker = usePreMarker;
		this.usePostMarker = usePostMarker;
		this.useTags = useTags;
		additionalContstruction();
		fillStatistics(c);
	}
	/**
	 * can be used to do additional construction before the constructor of this class starts creating statistics
	 */
	protected void additionalContstruction()
	{}
	/**
	 * this method fills all the maps with the given statistics, it goes over all the WordAndTagHolders
	 * by using the size and the addressing WordAndTagHolder-Structures via the getEntry method; sequences
	 * are mapped and then the frames and co-occurrences are mapped
	 * @param c
	 */
	private void fillStatistics(Corpus<WordAndTagHolder> c)
	{
		int dist = this.frame.length / 2;
		for(int i=0;i<c.size();++i)
		{
			WordAndTagHolder wta = c.getEntry(i);
			this.fillMappedSequence(wta);
			this.fillWordSequence(wta);
			for(int j=dist;j<this.wordSentence.size()-dist;++j)
			{addFrameAndWord(j);}
		}
	}
	/**
	 * gets the frame and word for a given position (relative to the possibly padded wordSentence) and
	 * then adds the necessary statistics
	 * @param pos the position being worked on
	 */
	private void addFrameAndWord(int pos)
	{
		fillFrame(pos);
		int fNum = getCode();
		int word = this.wordSentence.getInt(pos);
		addEndtryToMap(fNum, word,this.coCounts);
		addEndtryToMap(word, fNum, reverseCoCounts);
		this.wordCounts.addTo(word, 1.0);
		this.frameCounts.addTo(fNum, 1.0);
		if(pos > 0)
		{preWordStats(word,this.wordSentence.getInt(pos-1));}
		if(pos < wordSentence.size())
		{postWordStats(word,this.wordSentence.getInt(pos+1));}
	}
	/**
	 * @param firstKey
	 * @param secondKey
	 */
	protected void addEndtryToMap(int firstKey, int secondKey, Int2ObjectOpenHashMap<Int2DoubleOpenHashMap> to)
	{
		Int2DoubleOpenHashMap map = to.get(firstKey);
		if(map == null)
		{
			map = new Int2DoubleOpenHashMap();
			to.put(firstKey, map);
		}
		map.addTo(secondKey, 1.0);
	}
	/**
	 * can be overwritten to handle statistics on words that follow words, will only be called for words
	 * in frames and words that have at least a buffer preceding
	 * @param word
	 * @param integer
	 */
	protected void postWordStats(int word, int integer)
	{}
	/**
	 * can be overwritten to handle statistics on words that precede words, will only be called for words
	 * in frames and words that have at least a buffer preceding
	 * @param word
	 * @param integer
	 */
	protected void preWordStats(int word, Integer integer)
	{}
	/**
	 * returns the code for the given frame, if one exists; otherwise it gets a new code and does the necessary
	 * housekeeping of tracking the reverse of the frame
	 */
	private int getCode()
	{
		int num = this.frameNumbers.lookUp(frame);
		if(num < 0)
		{
			num = this.frameNumbers.assignNew(frame);
			this.reverseFrames.put(num, Arrays.copyOf(frame, frame.length));
		}
		return num;
	}
	/**
	 * this fills the frame with the information from the mappedSentence around the given position
	 * @param pos
	 */
	private void fillFrame(int pos)
	{
		int dist = this.frame.length / 2;
		int i = 0;
		for(;dist > 0;--dist)
		{this.frame[i++] = this.mappedSentence.getInt(pos-dist);}
		// this should actually only be necessary once, but I don't think it represents much of a performance hit
		this.frame[i++] = -2;
		int dist2 = this.frame.length / 2;
		for(dist=1;dist<=dist2;++dist)
		{this.frame[i++] = this.mappedSentence.getInt(pos+dist);}
	}
	/**
	 * set to true if tags should be used in the mappedSentence wherever possible
	 */
	private final boolean useTags;
	/**
	 * this fills the mappedSentence by putting in the pre-Makers, adding either the tags or words in sequence
	 * (dependent on the useTags variable) and then putting in the post-Markers
	 * @param wth
	 */
	public void fillMappedSequence(WordAndTagHolder wth)
	{
		this.mappedSentence.clear();
		for(int i=0;i<this.usePreMarker;++i)
		{this.mappedSentence.add(-1);}
		for(int i=0;i<wth.getLength();++i)
		{this.mappedSentence.add(useTags && wth.getTag(i) > 0 ? wth.getTag(i) : wth.getWord(i));}
		for(int i=0;i<this.usePostMarker;++i)
		{this.mappedSentence.add(-1);}
	}
	/**
	 * this fills the wordSentence with the appropriate pre-Markers, the words in the WordAndTagHolder and
	 * then the post-Markers
	 */
	public void fillWordSequence(WordAndTagHolder wth)
	{
		this.wordSentence.clear();
		for(int i=0;i<this.usePreMarker;++i)
		{this.wordSentence.add(-1);}
		for(int i=0;i<wth.getLength();++i)
		{this.wordSentence.add(wth.getWord(i));}
		for(int i=0;i<this.usePostMarker;++i)
		{this.wordSentence.add(-1);}
	}
	/**
	 * this method returns the number of times a word occurred in the corpus, note that only word occurrences within
	 * counted frames are used to compute the statistic
	 * @param word the code of the word for which statistics are requested
	 * @return the number of times the word occurred in a counted frame
	 */
	public double getWordCount(int word)
	{return this.wordCounts.get(word);}
	/**
	 * returns all the words that have occurred within a frame
	 * @return
	 */
	public IntCollection getAllWords()
	{return this.wordCounts.keySet();}
	/**
	 * returns true if this class would extract a frame for the given position
	 * @param position
	 * @return
	 */
	public boolean hasFrameFor(int position, WordAndTagHolder wth)
	{return ((position+this.usePreMarker-(this.frame.length/2)) >= 0) && (position-this.usePostMarker+(this.frame.length/2) < wth.getLength());}
	/**
	 * fills the frame used by the instance with the information from the last WordAndTagHolder read and for
	 * the given position, then returns the code for the frame that was generated
	 * @param pos
	 * @return
	 */
	public int fillFrameExternal(int pos)
	{
		this.fillFrame(pos+this.usePreMarker);
		return this.getCode();
	}
	/**
	 * returns a collection of all the frames for which this instance has created a statistics, coded
	 * by the numbers given by this instance
	 * @return
	 */
	public IntCollection getAllFrames()
	{return this.frameCounts.keySet();}
	/**
	 * returns the number of times the given frame (identified by the code generated by this instance) has
	 * occurred in the corpus used to create the statistics
	 * @param frame2
	 * @return
	 */
	public double getFrameFrequency(int of)
	{return this.frameCounts.get(of);}
	/**
	 * returns a collection of all the word codes the given frame (identified by one of the codes from this
	 * instance) co-occurred with
	 * @param of
	 * @return
	 */
	public IntCollection getCoOccurringWords(int of)
	{
		Int2DoubleOpenHashMap map = this.coCounts.get(of);
		if(map == null)
		{return null;}
		return map.keySet();
	}
	/**
	 * returns the number of times the given frame and word co-occurred, the frame is interpreted as a code
	 * from this instance, if there were no co-occurrences, then the returned value is 0.0
	 * @param frame2
	 * @param word
	 * @return
	 */
	public double getWordFrameFrequency(int frame2, int word)
	{
		Int2DoubleOpenHashMap map = this.coCounts.get(frame2);
		if(map == null)
		{return 0.0;}
		return map.get(word);
	}
	/**
	 * returns a string representation of the frame represented by the given code number according to this
	 * instance; all border markers are represented by 'BORDERMARKER' all is preceded by '<' and followed by '>';
	 * the left out element is replaced by '_'
	 * @param k
	 * @return
	 */
	public String frameToString(int k)
	{
		int[] reverse = this.reverseFrames.get(k);
		if(reverse == null)
		{return "<>";}
		StringBuilder sb = new StringBuilder("<");
		for(int i=0;i<reverse.length;++i)
		{
			sb.append(" ");
			int content = reverse[i];
			switch (content)
			{
			case -2:
				sb.append("_");
				break;
			case -1:
				sb.append("BORDERMARKER");
				break;
			default:
				sb.append(StringToInt.reverse(content));
				break;
			}
		}
		sb.append(" >");
		return sb.toString();
	}
	/**
	 * returns the actual word numbers frame that is associated to the given number, if there is no such frame,
	 * then null is returned
	 * @param frame2
	 * @return
	 */
	public int[] getFrameStructure(int of)
	{return this.reverseFrames.get(of);}
}